{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-19T07:36:18.035966Z",
     "start_time": "2017-08-19T07:36:18.030689Z"
    }
   },
   "source": [
    "# Data Exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T02:18:36.438988Z",
     "start_time": "2017-11-07T02:18:35.220392Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../lib')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "\n",
    "import tensorflow as tf\n",
    "import gensim\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T02:18:36.796440Z",
     "start_time": "2017-11-07T02:18:36.440252Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T02:18:36.937485Z",
     "start_time": "2017-11-07T02:18:36.797605Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: Qt5Agg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T02:18:36.941552Z",
     "start_time": "2017-11-07T02:18:36.938835Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load data frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T02:18:29.936865Z",
     "start_time": "2017-11-07T02:18:29.851948Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-720aca14c334>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHDFStore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'processed/stage1/data_frames.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtrain_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train_df'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtest_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'test_df'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "store = pd.HDFStore('processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']\n",
    "store.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:53:37.888075Z",
     "start_time": "2017-11-07T01:53:37.584607Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Class</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, imt, i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  Class       Gene Variation  \\\n",
       "0      0      3    [acsl4]   [r570s]   \n",
       "1      1      9    [naglu]   [p521l]   \n",
       "2      2      7      [pah]   [l333f]   \n",
       "3      3      2     [ing1]   [a148d]   \n",
       "4      4      9  [tmem216]    [g77a]   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[2, this, mutation, resulted, in, a, myelopro...  \n",
       "1  [[abstract, the, large, tumor, suppressor, 1, ...  \n",
       "2  [[vascular, endothelial, growth, factor, recep...  \n",
       "3  [[inflammatory, myofibroblastic, tumor, imt, i...  \n",
       "4  [[abstract, retinoblastoma, is, a, pediatric, ...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[chek2]</td>\n",
       "      <td>[h371y]</td>\n",
       "      <td>[[the, incidence, of, breast, cancer, is, incr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[axin2]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>[[an, unselected, series, of, 310, colorectal,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[wnt4]</td>\n",
       "      <td>[e216g]</td>\n",
       "      <td>[[mycosis, fungoides, and, szary, syndrome, ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[sucla2]</td>\n",
       "      <td>[g118r]</td>\n",
       "      <td>[[regulated, progression, through, the, cell, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[braf]</td>\n",
       "      <td>[t599instt]</td>\n",
       "      <td>[[pilocytic, astrocytoma, pa, is, emerging, as...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Gene                Variation  \\\n",
       "0   [chek2]                  [h371y]   \n",
       "1   [axin2]  [truncating, mutations]   \n",
       "2    [wnt4]                  [e216g]   \n",
       "3  [sucla2]                  [g118r]   \n",
       "4    [braf]              [t599instt]   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[the, incidence, of, breast, cancer, is, incr...  \n",
       "1  [[an, unselected, series, of, 310, colorectal,...  \n",
       "2  [[mycosis, fungoides, and, szary, syndrome, ar...  \n",
       "3  [[regulated, progression, through, the, cell, ...  \n",
       "4  [[pilocytic, astrocytoma, pa, is, emerging, as...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:53:37.891641Z",
     "start_time": "2017-11-07T01:53:37.889105Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8989\n",
      "986\n"
     ]
    }
   ],
   "source": [
    "print(len(train_df))\n",
    "print(len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:53:38.009548Z",
     "start_time": "2017-11-07T01:53:37.892945Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(364610, 364610)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_words, vocab_wordidx = None, None\n",
    "with open('processed/stage2/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (vocab_words, vocab_wordidx) = pickle.load(f)\n",
    "len(vocab_words), len(vocab_wordidx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:36.246856Z",
     "start_time": "2017-09-26T03:39:27.540818Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "350604"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_words = train_df.Sentences.apply(lambda ll: list(itertools.chain.from_iterable(ll)))\n",
    "train_words = list(itertools.chain.from_iterable(train_words))\n",
    "train_words = set(train_words)\n",
    "len(train_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:37.307783Z",
     "start_time": "2017-09-26T03:39:36.247959Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "173470"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_words = test_df.Sentences.apply(lambda ll: list(itertools.chain.from_iterable(ll)))\n",
    "test_words = list(itertools.chain.from_iterable(test_words))\n",
    "test_words = set(test_words)\n",
    "len(test_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:44.456081Z",
     "start_time": "2017-09-26T03:39:44.439394Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8632"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_variations = set(list(itertools.chain.from_iterable(train_df.Variation)))\n",
    "len(train_variations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:48.776740Z",
     "start_time": "2017-09-26T03:39:48.763744Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "951"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_variations = set(list(itertools.chain.from_iterable(test_df.Variation)))\n",
    "len(test_variations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:51.961285Z",
     "start_time": "2017-09-26T03:39:51.938393Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1507"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_genes = set(list(itertools.chain.from_iterable(train_df.Gene)))\n",
    "len(train_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:55.044967Z",
     "start_time": "2017-09-26T03:39:55.035227Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "279"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_genes = set(list(itertools.chain.from_iterable(test_df.Gene)))\n",
    "len(test_genes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "vocab_words and vocab_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:39:57.532369Z",
     "start_time": "2017-09-26T03:39:57.524901Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "265"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_genes & test_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:10.975479Z",
     "start_time": "2017-09-26T03:40:10.967020Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "814"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_variations & test_variations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:12.955707Z",
     "start_time": "2017-09-26T03:40:12.875766Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161141"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_words & test_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:27.198784Z",
     "start_time": "2017-09-26T03:40:27.179726Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8769 7760 768\n",
      "8769 2665 758\n"
     ]
    }
   ],
   "source": [
    "print(len(train_variations | test_variations), len(train_words & train_variations), len(train_words & test_variations))\n",
    "print(len(train_variations | test_variations), len(test_words & train_variations), len(test_words & test_variations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:36.911413Z",
     "start_time": "2017-09-26T03:40:36.901134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1521 769 199\n",
      "1521 616 188\n"
     ]
    }
   ],
   "source": [
    "print(len(train_genes | test_genes), len(train_words & train_genes), len(train_words & test_genes))\n",
    "print(len(train_genes | test_genes), len(test_words & train_genes), len(test_words & test_genes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:40.333946Z",
     "start_time": "2017-09-26T03:40:40.326528Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2',\n",
       " 'this',\n",
       " 'mutation',\n",
       " 'resulted',\n",
       " 'in',\n",
       " 'a',\n",
       " 'myeloproliferative',\n",
       " 'phenotype',\n",
       " 'including',\n",
       " 'erythrocytosis',\n",
       " 'in',\n",
       " 'a',\n",
       " 'murine',\n",
       " 'model',\n",
       " 'of',\n",
       " 'retroviral',\n",
       " 'bone',\n",
       " 'marrow',\n",
       " 'transplantation']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.Sentences[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:40:47.656633Z",
     "start_time": "2017-09-26T03:40:47.650220Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:41:01.633783Z",
     "start_time": "2017-09-26T03:41:01.619158Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2',\n",
       " 'this',\n",
       " 'mutation',\n",
       " 'resulted',\n",
       " 'in',\n",
       " 'a',\n",
       " 'myeloproliferative',\n",
       " 'phenotype',\n",
       " 'including',\n",
       " 'erythrocytosis',\n",
       " 'in',\n",
       " 'a',\n",
       " 'murine',\n",
       " 'model',\n",
       " 'of',\n",
       " 'retroviral',\n",
       " 'bone',\n",
       " 'marrow',\n",
       " 'transplantation']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_punctuations = [w for w in train_df.Sentences[0][0] if w not in string.punctuation]\n",
    "no_punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:41:06.571725Z",
     "start_time": "2017-09-26T03:41:06.545871Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8989.000000\n",
       "mean      324.372344\n",
       "std       217.407948\n",
       "min         1.000000\n",
       "25%       204.000000\n",
       "50%       283.000000\n",
       "75%       380.000000\n",
       "max      3119.000000\n",
       "Name: Sentences, dtype: float64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentence_counts = train_df.Sentences.apply(lambda document: len(document))\n",
    "train_sentence_counts.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:41:41.724107Z",
     "start_time": "2017-09-26T03:41:40.810751Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8989.000000\n",
       "mean       27.435461\n",
       "std         3.914051\n",
       "min         1.000000\n",
       "25%        24.972112\n",
       "50%        27.125000\n",
       "75%        29.482022\n",
       "max        52.466667\n",
       "Name: Sentences, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_words_in_sentences = train_df.Sentences.apply(lambda document: np.mean([len(sentence) for sentence in document]))\n",
    "train_words_in_sentences.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:41:45.401167Z",
     "start_time": "2017-09-26T03:41:45.375112Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8989.000000\n",
       "mean      324.372344\n",
       "std       217.407948\n",
       "min         1.000000\n",
       "25%       204.000000\n",
       "50%       283.000000\n",
       "75%       380.000000\n",
       "max      3119.000000\n",
       "Name: Sentences, dtype: float64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentences = train_df.Sentences.apply(lambda document: len(document))\n",
    "train_sentences.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:41:48.492573Z",
     "start_time": "2017-09-26T03:41:48.476923Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     986.000000\n",
       "mean      337.166329\n",
       "std       244.728149\n",
       "min         5.000000\n",
       "25%       195.000000\n",
       "50%       281.000000\n",
       "75%       401.000000\n",
       "max      2964.000000\n",
       "Name: Sentences, dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sentences = test_df.Sentences.apply(lambda document: len(document))\n",
    "test_sentences.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:42:19.789420Z",
     "start_time": "2017-09-26T03:41:52.831108Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8989.000000\n",
       "mean      151.113143\n",
       "std        22.861147\n",
       "min         4.000000\n",
       "25%       136.980435\n",
       "50%       149.089552\n",
       "75%       162.356401\n",
       "max       288.433333\n",
       "Name: Sentences, dtype: float64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_chars_in_sentences = train_df.Sentences.apply(lambda d: np.mean([np.sum([len(w) for w in s]) for s in d]))\n",
    "train_chars_in_sentences.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## encoding issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Data and classes exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:54:49.050141Z",
     "start_time": "2017-11-07T01:54:49.040657Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7    1692\n",
       "4    1397\n",
       "2    1193\n",
       "1     990\n",
       "6     983\n",
       "5     930\n",
       "3     739\n",
       "8     688\n",
       "9     377\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.Class.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:56:00.652668Z",
     "start_time": "2017-11-07T01:55:59.761603Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2       [[vascular, endothelial, growth, factor, recep...\n",
       "16      [[janus, jak, tyrosine, kinases, contain, a, t...\n",
       "25      [[the, ten-eleven, translocation, 1, tet1, gen...\n",
       "27      [[myc, expression, is, deregulated, in, a, wid...\n",
       "32      [[individuals, with, neurofibromatosis, type, ...\n",
       "48      [[this, 12-week, clinical, study, evaluated, t...\n",
       "87      [[quality, control, mechanisms, promote, aggre...\n",
       "93      [[c-kit, is, a, member, of, the, type, 3, subc...\n",
       "96      [[the, were, premature, closely, derived, to, ...\n",
       "101     [[kabuki, syndrome, ks, is, a, multiple, conge...\n",
       "108     [[eml4alk, fusions, define, a, subset, of, lun...\n",
       "111     [[e26, transformation-specific, ets, transcrip...\n",
       "113     [[activating, mutations, in, jak1, and, jak2, ...\n",
       "114     [[a, p53, hot-spot, mutation, found, frequentl...\n",
       "117     [[establishedmsh6-null, mice, present, afreque...\n",
       "133     [[recent, evidence, identified, a, genetic, an...\n",
       "155     [[ligand-induced, phosphorylation, of, the, re...\n",
       "157     [[introduction, anaplastic, lymphoma, kinase, ...\n",
       "165     [[kinase, domain, kd, mutations, of, bcr-abl, ...\n",
       "174     [[acquired, somatic, mutations, in, _atrx, an,...\n",
       "186     [[the, maintenance, of, genomic, integrity, du...\n",
       "187     [[inactivation, of, ras, gtpase, activating, p...\n",
       "191     [[the, oncogenic, property, of, anaplastic, ly...\n",
       "194     [[ezh2, enhancer, of, zeste, homolog, 2, is, a...\n",
       "203     [[acetylation, of, multiple, lysine, residues,...\n",
       "210     [[the, thyroid, trk-t3, oncogene, results, fro...\n",
       "212     [[interaction, of, tcf7l2, with, translocated,...\n",
       "225     [[personalized, therapy, provides, the, best, ...\n",
       "251     [[endometrial, stromal, sarcomas, ess, are, ge...\n",
       "254     [[extracellular, signal-regulated, kinase-1, a...\n",
       "                              ...                        \n",
       "8898    [[we, identified, novel, gene, fusions, in, pa...\n",
       "8902    [[the, congenital, fibrosarcoma, t, 12, 15, p1...\n",
       "8903    [[pediatric, high-grade, glioma, hgg, is, a, d...\n",
       "8905    [[pediatric, high-grade, glioma, hgg, is, a, d...\n",
       "8908    [[lung, cancer, is, the, leading, cause, of, c...\n",
       "8910    [[lung, cancer, is, the, leading, cause, of, c...\n",
       "8911    [[collagen, is, an, important, extracellular, ...\n",
       "8912    [[collagen, is, an, important, extracellular, ...\n",
       "8913    [[lung, cancer, is, the, leading, cause, of, c...\n",
       "8919    [[head, and, neck, squamous, cell, carcinoma, ...\n",
       "8925    [[rearrangements, of, the, proto-oncogene, ret...\n",
       "8926    [[the, ret, protooncogene, mutations, responsi...\n",
       "8927    [[we, investigatedthe, transformingactivityoft...\n",
       "8929    [[we, investigatedthe, transformingactivityoft...\n",
       "8930    [[ret, is, a, single-pass, transmembrane, rece...\n",
       "8931    [[somatic, ret, mutations, have, been, identie...\n",
       "8932    [[we, investigatedthe, transformingactivityoft...\n",
       "8933    [[we, investigatedthe, transformingactivityoft...\n",
       "8939    [[mutations, of, the, ret, receptor, tyrosine,...\n",
       "8946    [[ret, is, a, single-pass, transmembrane, rece...\n",
       "8947    [[activating, germ-line, point, mutations, in,...\n",
       "8952    [[a, considerable, proportion, of, ladcs, the,...\n",
       "8954    [[constitutive, activation, of, the, ret, rece...\n",
       "8956    [[we, investigatedthe, transformingactivityoft...\n",
       "8959    [[we, investigatedthe, transformingactivityoft...\n",
       "8960    [[we, investigatedthe, transformingactivityoft...\n",
       "8962    [[introduction, inherited, germ, line, activat...\n",
       "8964    [[the, ret, proto-oncogene, encodes, a, recept...\n",
       "8976    [[familial, platelet, disorder, with, propensi...\n",
       "8978    [[runx, proteins, belong, to, a, family, of, m...\n",
       "Name: Sentences, Length: 1692, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.Sentences[train_df.Class == 7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "827px",
    "left": "0px",
    "right": "1350px",
    "top": "52px",
    "width": "212px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
